#!/usr/bin/perl
BEGIN {
    unshift @INC, '../modules';
}
use warnings;
use strict;
use Lingua::EN::StopWords qw(%StopWords);
use Storable;
use HTML::Parser;
use DB_File;
use YAML;
use TFIDF qw(html2text html2tf html2tfidf tf2tfidf);

my %params = (
    working_dir   => '/home/xuqy/dmoz/data',
    dump_dir      => 'dump',
    min_page_len  => 1_000,
    max_page_len  => 128_000,
    min_idf_count => 10,
    tf_file       => 'tf',
    raw_idf_file  => 'idf.raw.db',
    idf_yml_file  => 'idf.yml',
    tfidf_file    => 'tfidf',
);
my $working_dir = $params{working_dir};
chdir $working_dir or die "$!";

my $verbose = 0;

$SIG{INT} = sub {
    $verbose = !$verbose;
};

my ( $max_page_len, $min_page_len, $min_idf_count ) =
  @params{'max_page_len', 'min_page_len', 'min_idf_count'};
my ( $tf_file, $raw_idf_file, $idf_yml_file, $tfidf_file ) =
  @params{ 'tf_file', 'raw_idf_file', 'idf_yml_file', 'tfidf_file' };

open OUT, "> $tf_file" or die "$!";
print "dump --> tf\n";

unlink $raw_idf_file if -e $raw_idf_file;
tie my %raw_idf, 'DB_File', $raw_idf_file or die "$!";

my $dump_dir = $params{dump_dir};
opendir DIR, $dump_dir or die "$!";
my $count = 0;
my $total = 0;
foreach my $entry ( readdir DIR ) {
    next if $entry =~ /^\.{1,2}$/;
    my $file = "$dump_dir/$entry";
    next if -s $file < $min_page_len or -s $file > $max_page_len;
    print $file, "\n" if $verbose;
    open my $in, '<', $file or die "$!";
    my $html = do { local $/; <$in> };
    my $tf_ref = html2tf($html);
    if ( %$tf_ref ) {
        $total++;
        foreach my $token ( sort keys %$tf_ref ) {
            print OUT $token, ' ', $tf_ref->{$token}, ' ';
            $raw_idf{$token}++;
        }
        print OUT "\n";
        $count++;
        print "Record: $count\n" unless $count % 1000;
    }
    close $in;
}

closedir DIR;
close OUT;
print "Done!\n";
print "Total tokens: ", scalar( keys %raw_idf ), "\n";

print "Shrinking idf...\n";
while ( my ( $token, $idf ) = each %raw_idf ) {
    delete $raw_idf{$token} unless $idf > $min_idf_count;
}
print "Left: ", scalar( keys %raw_idf ), "\n";

my $index = 1;
my %idf;
for my $token ( sort keys %raw_idf ) {
    my $value = $raw_idf{$token};
    $idf{$token} = [ $index, $value ];
    $index++;
}
untie %raw_idf;
$idf{TOTAL} = $total;
YAML::DumpFile( $params{idf_yml_file}, \%idf );

#unlink $raw_idf_file;
print "Done!\n";

print "idf --> tfidf\n";

open IN, "< $tf_file" or die "$!";
$count = 0;
open OUT, "> $tfidf_file" or die "$!";
while (<IN>) {
    my %tf = split /\s+/, $_;
    my $tfidf_ref = tf2tfidf( \%tf, \%idf, $total );
    if (@$tfidf_ref) {
        for my $entry (@$tfidf_ref) {
            print OUT sprintf( "%d:%f ", @$entry );
        }
        print OUT "\n";
        $count++;
        print "Record: $count\n" unless $count % 1000;
    }
}
print "Done!\n";

#unlink $tf_file;
close OUT;
print "All done!\n";

